Load and clean data

cloud_data_orig <- load_cloud_data()
cloud_data_ls <- clean_cloud_data(cloud_data_orig)
cloud_data <- dplyr::bind_rows(cloud_data_ls, .id = "image")

Exploratory Data Analysis

Raw Images

raw_features <- c("DF", "CF", "BF", "AF", "AN")
for (var in c("label", raw_features)) {
  cat(sprintf("\n\n### %s\n\n", var))
  plt <- plot_cloud_data(cloud_data, var)
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 10, fig_height = 4
  )
  subchunk_idx <- subchunk_idx + 1
}

label

DF

CF

BF

AF

AN

Correlations

for (image_id in unique(cloud_data$image)) {
  plt_df <- cloud_data |> 
    dplyr::filter(image == !!image_id)
  plt <- plot_pairs(
    plt_df, columns = c("DF", "CF", "BF", "AF", "AN"), 
    point_size = 0.1, subsample = 0.5, color = plt_df$label
  ) +
    ggplot2::scale_color_manual(values = cloud_colors) +
    ggplot2::scale_fill_manual(values = cloud_colors) +
    ggplot2::labs(title = sprintf("Image %s", image_id))
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 10, fig_height = 10
  )
  subchunk_idx <- subchunk_idx + 1
}

Feature Engineering

engineered_features <- c("NDAI", "SD", "CORR")
for (var in c("label", engineered_features)) {
  cat(sprintf("\n\n### %s\n\n", var))
  plt <- plot_cloud_data(cloud_data, var)
  vthemes::subchunkify(
    plt, i = subchunk_idx, fig_width = 10, fig_height = 4
  )
  subchunk_idx <- subchunk_idx + 1
}

label

NDAI

SD

CORR

Prediction Modeling

# save one image for testing
test_image_idx <- sample(unique(cloud_data$image), 1)
train_data_all <- cloud_data |>
  dplyr::filter(!(image %in% test_image_idx))
test_data_all <- cloud_data |>
  dplyr::filter(image %in% test_image_idx)
print(sprintf("Test image: %s", test_image_idx))
## [1] "Test image: 1"

Data Splitting using Random Sampling

First, let’s investigate what happens if we ignored the image structure and randomly sampled points into training and test sets. We will see that the estimated test accuracy (here, using logistic regression) is significantly higher than the observed test accuracy. In other words, we have overfitted to the training data. This holds true if we had used other prediction models, too.

# randomly sample points into training and test sets
nfolds <- 8
cv_foldids <- sample(1:nfolds, nrow(train_data_all), replace = TRUE)

# vector of predictor covariates
keep_vars <- c("DF", "CF", "BF", "AF", "AN", "binary_label")

# estimate test error for logistic regression
valid_auroc_ls <- list()
valid_preds_ls <- list()
for (fold in 1:nfolds) {
  # do data split
  cv_train_data_all <- train_data_all[cv_foldids != fold, ]
  cv_train_data <- cv_train_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  cv_valid_data_all <- train_data_all[cv_foldids == fold, ]
  cv_valid_data <- cv_valid_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  
  # fit and evaluate logistic regression
  fit <- glm(binary_label ~ ., data = cv_train_data, family = "binomial")
  valid_preds <- predict(fit, cv_valid_data, type = "response")
  log_auroc <- yardstick::roc_auc_vec(
    truth = cv_valid_data$binary_label, 
    estimate = valid_preds, 
    event_level = "second"
  )
  
  # save fold results for future investigation
  valid_preds_ls[[fold]] <- cv_valid_data_all |> 
    dplyr::mutate(
      yhat = valid_preds
    )
  valid_auroc_ls[[fold]] <- tibble::tibble(
    logistic = log_auroc
  )
}

# examine validation accuracy
valid_auroc_df <- dplyr::bind_rows(valid_auroc_ls, .id = "fold")
mean_valid_auroc_df <- valid_auroc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))

# test error for logistic regression
train_data <- train_data_all |> 
  dplyr::select(tidyselect::all_of(keep_vars))
test_data <- test_data_all |>
  dplyr::select(tidyselect::all_of(keep_vars))
fit <- glm(binary_label ~ ., data = train_data, family = "binomial")
test_preds <- predict(fit, test_data, type = "response")
test_auroc <- yardstick::roc_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)

tibble::tibble(
  `Estimated Test AUROC` = mean_valid_auroc_df$logistic,
  `Observed Test AUROC` = test_auroc
)
## # A tibble: 1 × 2
##   `Estimated Test AUROC` `Observed Test AUROC`
##                    <dbl>                 <dbl>
## 1                  0.814                 0.597

Data Splitting (Clustered Sampling)

To obtain a more accurate estimate of the test error, we can perform clustered sampling, where we partition the images into contiguous blocks when doing the data splitting. This better mimics the process of how we obtain our future data (that is, obtaining completely new images).

Image Blocks

Below, we show the image blocks used in the data splitting scheme.

# divide image into contiguous chunks
cloud_data <- add_cloud_blocks(cloud_data_ls)
plt <- plot_cloud_data(cloud_data, var = "block_id")
plt

Logistic Regression

By performing the data splitting scheme using clustered sampling, the estimated validation accuracy from logistic regression is now closer to the observed test error. This is because we are now training and evaluating our model on data that is more representative of the future data we will encounter.

train_data_all <- cloud_data |>
  dplyr::filter(!(image %in% test_image_idx))
test_data_all <- cloud_data |>
  dplyr::filter(image %in% test_image_idx)

# estimate test error for logistic regression
valid_auroc_ls <- list()
valid_preds_ls <- list()
for (fold in unique(train_data_all$block_id)) {
  # do data split
  cv_train_data_all <- train_data_all |> 
    dplyr::filter(block_id != !!fold)
  cv_train_data <- cv_train_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  cv_valid_data_all <- train_data_all |> 
    dplyr::filter(block_id == !!fold)
  cv_valid_data <- cv_valid_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  
  # fit and evaluate logistic regression
  fit <- glm(binary_label ~ ., data = cv_train_data, family = "binomial")
  valid_preds <- predict(fit, cv_valid_data, type = "response")
  log_auroc <- yardstick::roc_auc_vec(
    truth = cv_valid_data$binary_label, 
    estimate = valid_preds, 
    event_level = "second"
  )
  
  # save fold results for future investigation
  valid_preds_ls[[fold]] <- cv_valid_data_all |> 
    dplyr::mutate(
      yhat = valid_preds
    )
  valid_auroc_ls[[fold]] <- tibble::tibble(
    logistic = log_auroc
  )
}

# examine validation accuracy
valid_auroc_df <- dplyr::bind_rows(valid_auroc_ls, .id = "fold")
mean_valid_auroc_df <- valid_auroc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))

tibble::tibble(
  `Estimated Test AUROC` = mean_valid_auroc_df$logistic,
  `Observed Test AUROC` = test_auroc
)
## # A tibble: 1 × 2
##   `Estimated Test AUROC` `Observed Test AUROC`
##                    <dbl>                 <dbl>
## 1                  0.655                 0.597

Multiple Methods

We next provide an example of how to perform the data splitting scheme when trying to both tune hyperparameters in models and selecting the best model.

Models under consideration:

  • Logistic regression
  • Lasso regression (needs tuning)
  • Ridge regression (needs tuning)
  • Random forest (no tuning; using default hyperparameters)

Note that within glmnet::cv.glmnet, there is an interior cross-validation loop to tune the hyperparameters for LASSO and ridge, and by explicitly setting the foldid, we ensure that the cross-validation is done using our clustered sampling scheme by image block. If the R function doesn’t have a built-in CV option, we would need to code this up ourselves (or using other functions like from the caret R package).

train_data <- cloud_data |>
  dplyr::filter(!(image %in% test_image_idx))
test_data <- cloud_data |>
  dplyr::filter(image %in% test_image_idx)

# evaluate validation error for various models
valid_auroc_ls <- list()
valid_preds_ls <- list()
for (fold in unique(train_data_all$block_id)) {
  # do data split
  cv_train_data_all <- train_data_all |> 
    dplyr::filter(block_id != !!fold)
  cv_train_data <- cv_train_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  cv_valid_data_all <- train_data_all |> 
    dplyr::filter(block_id == !!fold)
  cv_valid_data <- cv_valid_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  
  # fit logistic regression
  log_fit <- glm(
    binary_label ~ ., data = cv_train_data, family = "binomial"
  )
  log_preds <- predict(log_fit, cv_valid_data, type = "response")
  
  # fit and evaluate lasso regression
  lasso_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 1,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  lasso_preds <- predict(
    lasso_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit and evaluate ridge regression
  ridge_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 0,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  ridge_preds <- predict(
    ridge_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit random forest
  rf_fit <- ranger::ranger(
    binary_label ~ ., data = cv_train_data, 
    probability = TRUE, verbose = FALSE
  )
  rf_preds <- predict(rf_fit, cv_valid_data)$predictions[, 2]
  
  # evaluate predictions
  preds_ls <- list(
    "logistic" = log_preds,
    "lasso" = lasso_preds,
    "ridge" = ridge_preds,
    "rf" = rf_preds
  )
  valid_auroc_ls[[fold]] <- purrr::map(
    preds_ls,
    ~ yardstick::roc_auc_vec(
      truth = cv_valid_data$binary_label, 
      estimate = c(.x), 
      event_level = "second"
    )
  ) |>
    dplyr::bind_rows(.id = "method")
  
  # save fold predictions for future investigation
  valid_preds_ls[[fold]] <- cv_valid_data_all |> 
    dplyr::bind_cols(preds_ls)
}

# examine validation accuracy
valid_auroc_df <- dplyr::bind_rows(valid_auroc_ls, .id = "fold")
mean_valid_auroc_df <- valid_auroc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))

# evaluate best model on test set
train_data <- train_data_all |> 
  dplyr::select(tidyselect::all_of(keep_vars))
test_data <- test_data_all |>
  dplyr::select(tidyselect::all_of(keep_vars))
best_fit <- ranger::ranger(
  binary_label ~ ., data = train_data, probability = TRUE, verbose = FALSE
)
test_preds <- predict(best_fit, test_data)$predictions[, 2]
test_auroc <- yardstick::roc_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)

err_df <- mean_valid_auroc_df |>
  dplyr::rename_with(~ sprintf("Validation %s AUROC", stringr::str_to_title(.x))) |> 
  dplyr::mutate(
    `Test AUROC` = test_auroc
  )

vthemes::pretty_kable(
  valid_auroc_df, caption = "Fold-wise Validation AUROC for Various Models"
)
Table 1: Fold-wise Validation AUROC for Various Models
fold logistic lasso ridge rf
6 0.885 0.885 0.886 0.881
5 0.791 0.798 0.811 0.713
7 0.817 0.814 0.815 0.850
8 0.907 0.908 0.912 0.927
10 0.378 0.378 0.382 0.494
9 0.681 0.681 0.678 0.578
11 0.130 0.129 0.124 0.608
12 0.652 0.653 0.655 0.843 
vthemes::pretty_kable(
  err_df, caption = "Overall Prediction Performance"
)
Table 1: Overall Prediction Performance
Validation Logistic AUROC Validation Lasso AUROC Validation Ridge AUROC Validation Rf AUROC Test AUROC
0.655 0.656 0.658 0.737 0.759

Including Engineered Features

Engineering features based upon prior or domain knowledge can greatly improve the accuracy of our models. In Shi et al. (2008), the authors engineered three additional features:

  • NDAI: measures difference in reflectance values between different radiance angles (or spectral bands)
  • CORR: measures correlation between the different radiance angles
  • SD: measures variability in reflectance values surrounding the pixel
keep_vars <- c("NDAI", "SD", "CORR", keep_vars)

# evaluate validation error for various models
valid_auroc_ls <- list()
valid_preds_ls <- list()
for (fold in unique(train_data_all$block_id)) {
  # do data split
  cv_train_data_all <- train_data_all |> 
    dplyr::filter(block_id != !!fold)
  cv_train_data <- cv_train_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  cv_valid_data_all <- train_data_all |> 
    dplyr::filter(block_id == !!fold)
  cv_valid_data <- cv_valid_data_all |> 
    dplyr::select(tidyselect::all_of(keep_vars))
  
  # fit logistic regression
  log_fit <- glm(
    binary_label ~ ., data = cv_train_data, family = "binomial"
  )
  log_preds <- predict(log_fit, cv_valid_data, type = "response")
  
  # fit and evaluate lasso regression
  lasso_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 1,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  lasso_preds <- predict(
    lasso_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit and evaluate ridge regression
  ridge_fit <- glmnet::cv.glmnet(
    x = as.matrix(cv_train_data |> dplyr::select(-binary_label)),
    y = cv_train_data$binary_label,
    family = "binomial",
    alpha = 0,
    foldid = as.numeric(as.factor(cv_train_data_all$block_id))
  )
  ridge_preds <- predict(
    ridge_fit, 
    as.matrix(cv_valid_data |> dplyr::select(-binary_label)), 
    s = "lambda.min", # or "lambda.1se"
    type = "response"
  )
  
  # fit random forest
  rf_fit <- ranger::ranger(
    binary_label ~ ., data = cv_train_data, 
    probability = TRUE, verbose = FALSE
  )
  rf_preds <- predict(rf_fit, cv_valid_data)$predictions[, 2]
  
  # evaluate predictions
  preds_ls <- list(
    "logistic" = log_preds,
    "lasso" = lasso_preds,
    "ridge" = ridge_preds,
    "rf" = rf_preds
  )
  valid_auroc_ls[[fold]] <- purrr::map(
    preds_ls,
    ~ yardstick::roc_auc_vec(
      truth = cv_valid_data$binary_label, 
      estimate = c(.x), 
      event_level = "second"
    )
  ) |>
    dplyr::bind_rows(.id = "method")
  
  # save fold predictions for future investigation
  valid_preds_ls[[fold]] <- cv_valid_data_all |> 
    dplyr::bind_cols(preds_ls)
}

# examine validation accuracy
valid_auroc_df <- dplyr::bind_rows(valid_auroc_ls, .id = "fold")
mean_valid_auroc_df <- valid_auroc_df |> 
  dplyr::summarise(dplyr::across(-fold, ~ mean(.x, na.rm = TRUE)))

# evaluate best model on test set
train_data <- train_data_all |> 
  dplyr::select(tidyselect::all_of(keep_vars))
test_data <- test_data_all |>
  dplyr::select(tidyselect::all_of(keep_vars))
best_fit <- ranger::ranger(
  binary_label ~ ., data = train_data, probability = TRUE, verbose = FALSE
)
test_preds <- predict(best_fit, test_data)$predictions[, 2]
test_auroc <- yardstick::roc_auc_vec(
  truth = test_data$binary_label, 
  estimate = test_preds, 
  event_level = "second"
)

new_err_df <- mean_valid_auroc_df |>
  dplyr::rename_with(~ sprintf("Validation %s AUROC", stringr::str_to_title(.x))) |> 
  dplyr::mutate(
    `Test AUROC` = test_auroc
  )
new_err_df <- list(
  "Raw Features Only" = err_df,
  "Including Engineered Features" = new_err_df
) |> 
  dplyr::bind_rows(.id = "Feature Set")

vthemes::pretty_kable(
  valid_auroc_df, caption = "Fold-wise Validation AUROC for Various Models"
)
Table 2: Fold-wise Validation AUROC for Various Models
fold logistic lasso ridge rf
6 0.875 0.875 0.880 0.912
5 0.820 0.821 0.827 0.791
7 0.876 0.874 0.862 0.919
8 0.945 0.945 0.955 0.951
10 0.430 0.427 0.383 0.634
9 0.732 0.733 0.726 0.673
11 0.342 0.327 0.244 0.834
12 0.788 0.786 0.763 0.899 
vthemes::pretty_kable(new_err_df)
Table 2:
Feature Set Validation Logistic AUROC Validation Lasso AUROC Validation Ridge AUROC Validation Rf AUROC Test AUROC
Raw Features Only 0.655 0.656 0.658 0.737 0.759
Including Engineered Features 0.726 0.724 0.705 0.826 0.823

Post-hoc investigations

Let’s look at the held-out folds where the methods didn’t perform so well and compare the predictions across methods.

fold <- "10"
preds_df <- valid_preds_ls[[fold]]
plot_vars <- c(
  "binary_label", "logistic", "lasso", "ridge", "rf",
  "DF", "CF", "BF", "AF", "AN", "NDAI", "SD", "CORR"
)
plt_ls <- list()
for (var in plot_vars) {
  plt_ls[[var]] <- plot_cloud_data(preds_df, var)
}
plt <- patchwork::wrap_plots(plt_ls, ncol = 5)
plt

# plt <- plot_pairs(
#   preds_df, columns = c("logistic", "lasso", "ridge", "rf"), 
#   point_size = 0.1, color = preds_df$binary_label
# ) +
#   ggplot2::scale_color_manual(values = cloud_colors) +
#   ggplot2::scale_fill_manual(values = cloud_colors
# )
# plt

Interpretations

To be continued…

LS0tCnRpdGxlOiAiU3VwZXJ2aXNlZCBMZWFybmluZyB3aXRoIENsb3VkcyBEYXRhIgphdXRob3I6ICJUaWZmYW55IFRhbmciCmRhdGU6ICJgciBTeXMuRGF0ZSgpYCIKb3V0cHV0OiAKICB2dGhlbWVzOjp2bW9kZXJuOgogICAgY29kZV9mb2xkaW5nOiBzaG93Ci0tLQoKYGBge3Igc2V0dXAsIGluY2x1ZGU9RkFMU0V9CmtuaXRyOjpvcHRzX2NodW5rJHNldCgKICBlY2hvID0gVFJVRSwKICB3YXJuaW5nID0gRkFMU0UsCiAgbWVzc2FnZSA9IEZBTFNFCikKCnNvdXJjZShoZXJlOjpoZXJlKCJSIiwgImxvYWQuUiIpKQpzb3VyY2UoaGVyZTo6aGVyZSgiUiIsICJjbGVhbi5SIikpCnNvdXJjZShoZXJlOjpoZXJlKCJSIiwgInBsb3QuUiIpKQoKc3ViY2h1bmtfaWR4IDwtIDEKc2V0LnNlZWQoMjQyKQoKY2xvdWRfY29sb3JzIDwtIGMoImRhcmsgZ3JlZW4iLCAibGlnaHQgYmx1ZSIsICJibGFjayIpCmBgYAoKIyBMb2FkIGFuZCBjbGVhbiBkYXRhIHsudGFic2V0IC50YWJzZXQtdm1vZGVybn0KCmBgYHtyfQpjbG91ZF9kYXRhX29yaWcgPC0gbG9hZF9jbG91ZF9kYXRhKCkKY2xvdWRfZGF0YV9scyA8LSBjbGVhbl9jbG91ZF9kYXRhKGNsb3VkX2RhdGFfb3JpZykKY2xvdWRfZGF0YSA8LSBkcGx5cjo6YmluZF9yb3dzKGNsb3VkX2RhdGFfbHMsIC5pZCA9ICJpbWFnZSIpCmBgYAoKIyBFeHBsb3JhdG9yeSBEYXRhIEFuYWx5c2lzIHsudGFic2V0IC50YWJzZXQtdm1vZGVybn0KCiMjIFJhdyBJbWFnZXMgey50YWJzZXQgLnRhYnNldC1waWxscyAudGFic2V0LXNxdWFyZX0KCmBgYHtyIHJlc3VsdHMgPSAiYXNpcyJ9CnJhd19mZWF0dXJlcyA8LSBjKCJERiIsICJDRiIsICJCRiIsICJBRiIsICJBTiIpCmZvciAodmFyIGluIGMoImxhYmVsIiwgcmF3X2ZlYXR1cmVzKSkgewogIGNhdChzcHJpbnRmKCJcblxuIyMjICVzXG5cbiIsIHZhcikpCiAgcGx0IDwtIHBsb3RfY2xvdWRfZGF0YShjbG91ZF9kYXRhLCB2YXIpCiAgdnRoZW1lczo6c3ViY2h1bmtpZnkoCiAgICBwbHQsIGkgPSBzdWJjaHVua19pZHgsIGZpZ193aWR0aCA9IDEwLCBmaWdfaGVpZ2h0ID0gNAogICkKICBzdWJjaHVua19pZHggPC0gc3ViY2h1bmtfaWR4ICsgMQp9CmBgYAoKIyMgQ29ycmVsYXRpb25zIHsudGFic2V0IC50YWJzZXQtcGlsbHMgLnRhYnNldC1zcXVhcmV9CgpgYGB7ciByZXN1bHRzID0gImFzaXMifQpmb3IgKGltYWdlX2lkIGluIHVuaXF1ZShjbG91ZF9kYXRhJGltYWdlKSkgewogIHBsdF9kZiA8LSBjbG91ZF9kYXRhIHw+IAogICAgZHBseXI6OmZpbHRlcihpbWFnZSA9PSAhIWltYWdlX2lkKQogIHBsdCA8LSBwbG90X3BhaXJzKAogICAgcGx0X2RmLCBjb2x1bW5zID0gYygiREYiLCAiQ0YiLCAiQkYiLCAiQUYiLCAiQU4iKSwgCiAgICBwb2ludF9zaXplID0gMC4xLCBzdWJzYW1wbGUgPSAwLjUsIGNvbG9yID0gcGx0X2RmJGxhYmVsCiAgKSArCiAgICBnZ3Bsb3QyOjpzY2FsZV9jb2xvcl9tYW51YWwodmFsdWVzID0gY2xvdWRfY29sb3JzKSArCiAgICBnZ3Bsb3QyOjpzY2FsZV9maWxsX21hbnVhbCh2YWx1ZXMgPSBjbG91ZF9jb2xvcnMpICsKICAgIGdncGxvdDI6OmxhYnModGl0bGUgPSBzcHJpbnRmKCJJbWFnZSAlcyIsIGltYWdlX2lkKSkKICB2dGhlbWVzOjpzdWJjaHVua2lmeSgKICAgIHBsdCwgaSA9IHN1YmNodW5rX2lkeCwgZmlnX3dpZHRoID0gMTAsIGZpZ19oZWlnaHQgPSAxMAogICkKICBzdWJjaHVua19pZHggPC0gc3ViY2h1bmtfaWR4ICsgMQp9CmBgYAoKIyMgRmVhdHVyZSBFbmdpbmVlcmluZyB7LnRhYnNldCAudGFic2V0LXBpbGxzIC50YWJzZXQtc3F1YXJlfQoKYGBge3IgcmVzdWx0cyA9ICJhc2lzIn0KZW5naW5lZXJlZF9mZWF0dXJlcyA8LSBjKCJOREFJIiwgIlNEIiwgIkNPUlIiKQpmb3IgKHZhciBpbiBjKCJsYWJlbCIsIGVuZ2luZWVyZWRfZmVhdHVyZXMpKSB7CiAgY2F0KHNwcmludGYoIlxuXG4jIyMgJXNcblxuIiwgdmFyKSkKICBwbHQgPC0gcGxvdF9jbG91ZF9kYXRhKGNsb3VkX2RhdGEsIHZhcikKICB2dGhlbWVzOjpzdWJjaHVua2lmeSgKICAgIHBsdCwgaSA9IHN1YmNodW5rX2lkeCwgZmlnX3dpZHRoID0gMTAsIGZpZ19oZWlnaHQgPSA0CiAgKQogIHN1YmNodW5rX2lkeCA8LSBzdWJjaHVua19pZHggKyAxCn0KYGBgCgojIFByZWRpY3Rpb24gTW9kZWxpbmcgey50YWJzZXQgLnRhYnNldC12bW9kZXJufQoKYGBge3J9CiMgc2F2ZSBvbmUgaW1hZ2UgZm9yIHRlc3RpbmcKdGVzdF9pbWFnZV9pZHggPC0gc2FtcGxlKHVuaXF1ZShjbG91ZF9kYXRhJGltYWdlKSwgMSkKdHJhaW5fZGF0YV9hbGwgPC0gY2xvdWRfZGF0YSB8PgogIGRwbHlyOjpmaWx0ZXIoIShpbWFnZSAlaW4lIHRlc3RfaW1hZ2VfaWR4KSkKdGVzdF9kYXRhX2FsbCA8LSBjbG91ZF9kYXRhIHw+CiAgZHBseXI6OmZpbHRlcihpbWFnZSAlaW4lIHRlc3RfaW1hZ2VfaWR4KQpwcmludChzcHJpbnRmKCJUZXN0IGltYWdlOiAlcyIsIHRlc3RfaW1hZ2VfaWR4KSkKYGBgCgojIyBEYXRhIFNwbGl0dGluZyB1c2luZyBSYW5kb20gU2FtcGxpbmcKCjxkaXYgY2xhc3M9InBhbmVsIHBhbmVsLWRlZmF1bHQgcGFkZGVkLXBhbmVsIj4KRmlyc3QsIGxldCdzIGludmVzdGlnYXRlIHdoYXQgaGFwcGVucyBpZiB3ZSBpZ25vcmVkIHRoZSBpbWFnZSBzdHJ1Y3R1cmUgYW5kIHJhbmRvbWx5IHNhbXBsZWQgcG9pbnRzIGludG8gdHJhaW5pbmcgYW5kIHRlc3Qgc2V0cy4gV2Ugd2lsbCBzZWUgdGhhdCB0aGUgZXN0aW1hdGVkIHRlc3QgYWNjdXJhY3kgKGhlcmUsIHVzaW5nIGxvZ2lzdGljIHJlZ3Jlc3Npb24pIGlzIHNpZ25pZmljYW50bHkgaGlnaGVyIHRoYW4gdGhlIG9ic2VydmVkIHRlc3QgYWNjdXJhY3kuIEluIG90aGVyIHdvcmRzLCB3ZSBoYXZlIG92ZXJmaXR0ZWQgdG8gdGhlIHRyYWluaW5nIGRhdGEuIFRoaXMgaG9sZHMgdHJ1ZSBpZiB3ZSBoYWQgdXNlZCBvdGhlciBwcmVkaWN0aW9uIG1vZGVscywgdG9vLgo8L2Rpdj4KCmBgYHtyfQojIHJhbmRvbWx5IHNhbXBsZSBwb2ludHMgaW50byB0cmFpbmluZyBhbmQgdGVzdCBzZXRzCm5mb2xkcyA8LSA4CmN2X2ZvbGRpZHMgPC0gc2FtcGxlKDE6bmZvbGRzLCBucm93KHRyYWluX2RhdGFfYWxsKSwgcmVwbGFjZSA9IFRSVUUpCgojIHZlY3RvciBvZiBwcmVkaWN0b3IgY292YXJpYXRlcwprZWVwX3ZhcnMgPC0gYygiREYiLCAiQ0YiLCAiQkYiLCAiQUYiLCAiQU4iLCAiYmluYXJ5X2xhYmVsIikKCiMgZXN0aW1hdGUgdGVzdCBlcnJvciBmb3IgbG9naXN0aWMgcmVncmVzc2lvbgp2YWxpZF9hdXJvY19scyA8LSBsaXN0KCkKdmFsaWRfcHJlZHNfbHMgPC0gbGlzdCgpCmZvciAoZm9sZCBpbiAxOm5mb2xkcykgewogICMgZG8gZGF0YSBzcGxpdAogIGN2X3RyYWluX2RhdGFfYWxsIDwtIHRyYWluX2RhdGFfYWxsW2N2X2ZvbGRpZHMgIT0gZm9sZCwgXQogIGN2X3RyYWluX2RhdGEgPC0gY3ZfdHJhaW5fZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQogIGN2X3ZhbGlkX2RhdGFfYWxsIDwtIHRyYWluX2RhdGFfYWxsW2N2X2ZvbGRpZHMgPT0gZm9sZCwgXQogIGN2X3ZhbGlkX2RhdGEgPC0gY3ZfdmFsaWRfZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQogIAogICMgZml0IGFuZCBldmFsdWF0ZSBsb2dpc3RpYyByZWdyZXNzaW9uCiAgZml0IDwtIGdsbShiaW5hcnlfbGFiZWwgfiAuLCBkYXRhID0gY3ZfdHJhaW5fZGF0YSwgZmFtaWx5ID0gImJpbm9taWFsIikKICB2YWxpZF9wcmVkcyA8LSBwcmVkaWN0KGZpdCwgY3ZfdmFsaWRfZGF0YSwgdHlwZSA9ICJyZXNwb25zZSIpCiAgbG9nX2F1cm9jIDwtIHlhcmRzdGljazo6cm9jX2F1Y192ZWMoCiAgICB0cnV0aCA9IGN2X3ZhbGlkX2RhdGEkYmluYXJ5X2xhYmVsLCAKICAgIGVzdGltYXRlID0gdmFsaWRfcHJlZHMsIAogICAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgogICkKICAKICAjIHNhdmUgZm9sZCByZXN1bHRzIGZvciBmdXR1cmUgaW52ZXN0aWdhdGlvbgogIHZhbGlkX3ByZWRzX2xzW1tmb2xkXV0gPC0gY3ZfdmFsaWRfZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6bXV0YXRlKAogICAgICB5aGF0ID0gdmFsaWRfcHJlZHMKICAgICkKICB2YWxpZF9hdXJvY19sc1tbZm9sZF1dIDwtIHRpYmJsZTo6dGliYmxlKAogICAgbG9naXN0aWMgPSBsb2dfYXVyb2MKICApCn0KCiMgZXhhbWluZSB2YWxpZGF0aW9uIGFjY3VyYWN5CnZhbGlkX2F1cm9jX2RmIDwtIGRwbHlyOjpiaW5kX3Jvd3ModmFsaWRfYXVyb2NfbHMsIC5pZCA9ICJmb2xkIikKbWVhbl92YWxpZF9hdXJvY19kZiA8LSB2YWxpZF9hdXJvY19kZiB8PiAKICBkcGx5cjo6c3VtbWFyaXNlKGRwbHlyOjphY3Jvc3MoLWZvbGQsIH4gbWVhbigueCwgbmEucm0gPSBUUlVFKSkpCgojIHRlc3QgZXJyb3IgZm9yIGxvZ2lzdGljIHJlZ3Jlc3Npb24KdHJhaW5fZGF0YSA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQp0ZXN0X2RhdGEgPC0gdGVzdF9kYXRhX2FsbCB8PgogIGRwbHlyOjpzZWxlY3QodGlkeXNlbGVjdDo6YWxsX29mKGtlZXBfdmFycykpCmZpdCA8LSBnbG0oYmluYXJ5X2xhYmVsIH4gLiwgZGF0YSA9IHRyYWluX2RhdGEsIGZhbWlseSA9ICJiaW5vbWlhbCIpCnRlc3RfcHJlZHMgPC0gcHJlZGljdChmaXQsIHRlc3RfZGF0YSwgdHlwZSA9ICJyZXNwb25zZSIpCnRlc3RfYXVyb2MgPC0geWFyZHN0aWNrOjpyb2NfYXVjX3ZlYygKICB0cnV0aCA9IHRlc3RfZGF0YSRiaW5hcnlfbGFiZWwsIAogIGVzdGltYXRlID0gdGVzdF9wcmVkcywgCiAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgopCgp0aWJibGU6OnRpYmJsZSgKICBgRXN0aW1hdGVkIFRlc3QgQVVST0NgID0gbWVhbl92YWxpZF9hdXJvY19kZiRsb2dpc3RpYywKICBgT2JzZXJ2ZWQgVGVzdCBBVVJPQ2AgPSB0ZXN0X2F1cm9jCikKYGBgCgojIyBEYXRhIFNwbGl0dGluZyAoQ2x1c3RlcmVkIFNhbXBsaW5nKSB7LnRhYnNldCAudGFic2V0LXBpbGxzIC50YWJzZXQtc3F1YXJlfQoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpUbyBvYnRhaW4gYSBtb3JlIGFjY3VyYXRlIGVzdGltYXRlIG9mIHRoZSB0ZXN0IGVycm9yLCB3ZSBjYW4gcGVyZm9ybSBjbHVzdGVyZWQgc2FtcGxpbmcsIHdoZXJlIHdlIHBhcnRpdGlvbiB0aGUgaW1hZ2VzIGludG8gY29udGlndW91cyBibG9ja3Mgd2hlbiBkb2luZyB0aGUgZGF0YSBzcGxpdHRpbmcuIFRoaXMgYmV0dGVyIG1pbWljcyB0aGUgcHJvY2VzcyBvZiBob3cgd2Ugb2J0YWluIG91ciBmdXR1cmUgZGF0YSAodGhhdCBpcywgb2J0YWluaW5nIGNvbXBsZXRlbHkgbmV3IGltYWdlcykuCjwvZGl2PgoKIyMjIEltYWdlIEJsb2NrcwoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpCZWxvdywgd2Ugc2hvdyB0aGUgaW1hZ2UgYmxvY2tzIHVzZWQgaW4gdGhlIGRhdGEgc3BsaXR0aW5nIHNjaGVtZS4KPC9kaXY+CgpgYGB7cn0KIyBkaXZpZGUgaW1hZ2UgaW50byBjb250aWd1b3VzIGNodW5rcwpjbG91ZF9kYXRhIDwtIGFkZF9jbG91ZF9ibG9ja3MoY2xvdWRfZGF0YV9scykKcGx0IDwtIHBsb3RfY2xvdWRfZGF0YShjbG91ZF9kYXRhLCB2YXIgPSAiYmxvY2tfaWQiKQpwbHQKYGBgCgojIyMgTG9naXN0aWMgUmVncmVzc2lvbgoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpCeSBwZXJmb3JtaW5nIHRoZSBkYXRhIHNwbGl0dGluZyBzY2hlbWUgdXNpbmcgY2x1c3RlcmVkIHNhbXBsaW5nLCB0aGUgZXN0aW1hdGVkIHZhbGlkYXRpb24gYWNjdXJhY3kgZnJvbSBsb2dpc3RpYyByZWdyZXNzaW9uIGlzIG5vdyBjbG9zZXIgdG8gdGhlIG9ic2VydmVkIHRlc3QgZXJyb3IuIFRoaXMgaXMgYmVjYXVzZSB3ZSBhcmUgbm93IHRyYWluaW5nIGFuZCBldmFsdWF0aW5nIG91ciBtb2RlbCBvbiBkYXRhIHRoYXQgaXMgbW9yZSByZXByZXNlbnRhdGl2ZSBvZiB0aGUgZnV0dXJlIGRhdGEgd2Ugd2lsbCBlbmNvdW50ZXIuCjwvZGl2PgoKYGBge3J9CnRyYWluX2RhdGFfYWxsIDwtIGNsb3VkX2RhdGEgfD4KICBkcGx5cjo6ZmlsdGVyKCEoaW1hZ2UgJWluJSB0ZXN0X2ltYWdlX2lkeCkpCnRlc3RfZGF0YV9hbGwgPC0gY2xvdWRfZGF0YSB8PgogIGRwbHlyOjpmaWx0ZXIoaW1hZ2UgJWluJSB0ZXN0X2ltYWdlX2lkeCkKCiMgZXN0aW1hdGUgdGVzdCBlcnJvciBmb3IgbG9naXN0aWMgcmVncmVzc2lvbgp2YWxpZF9hdXJvY19scyA8LSBsaXN0KCkKdmFsaWRfcHJlZHNfbHMgPC0gbGlzdCgpCmZvciAoZm9sZCBpbiB1bmlxdWUodHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKSB7CiAgIyBkbyBkYXRhIHNwbGl0CiAgY3ZfdHJhaW5fZGF0YV9hbGwgPC0gdHJhaW5fZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6ZmlsdGVyKGJsb2NrX2lkICE9ICEhZm9sZCkKICBjdl90cmFpbl9kYXRhIDwtIGN2X3RyYWluX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OnNlbGVjdCh0aWR5c2VsZWN0OjphbGxfb2Yoa2VlcF92YXJzKSkKICBjdl92YWxpZF9kYXRhX2FsbCA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpmaWx0ZXIoYmxvY2tfaWQgPT0gISFmb2xkKQogIGN2X3ZhbGlkX2RhdGEgPC0gY3ZfdmFsaWRfZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQogIAogICMgZml0IGFuZCBldmFsdWF0ZSBsb2dpc3RpYyByZWdyZXNzaW9uCiAgZml0IDwtIGdsbShiaW5hcnlfbGFiZWwgfiAuLCBkYXRhID0gY3ZfdHJhaW5fZGF0YSwgZmFtaWx5ID0gImJpbm9taWFsIikKICB2YWxpZF9wcmVkcyA8LSBwcmVkaWN0KGZpdCwgY3ZfdmFsaWRfZGF0YSwgdHlwZSA9ICJyZXNwb25zZSIpCiAgbG9nX2F1cm9jIDwtIHlhcmRzdGljazo6cm9jX2F1Y192ZWMoCiAgICB0cnV0aCA9IGN2X3ZhbGlkX2RhdGEkYmluYXJ5X2xhYmVsLCAKICAgIGVzdGltYXRlID0gdmFsaWRfcHJlZHMsIAogICAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgogICkKICAKICAjIHNhdmUgZm9sZCByZXN1bHRzIGZvciBmdXR1cmUgaW52ZXN0aWdhdGlvbgogIHZhbGlkX3ByZWRzX2xzW1tmb2xkXV0gPC0gY3ZfdmFsaWRfZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6bXV0YXRlKAogICAgICB5aGF0ID0gdmFsaWRfcHJlZHMKICAgICkKICB2YWxpZF9hdXJvY19sc1tbZm9sZF1dIDwtIHRpYmJsZTo6dGliYmxlKAogICAgbG9naXN0aWMgPSBsb2dfYXVyb2MKICApCn0KCiMgZXhhbWluZSB2YWxpZGF0aW9uIGFjY3VyYWN5CnZhbGlkX2F1cm9jX2RmIDwtIGRwbHlyOjpiaW5kX3Jvd3ModmFsaWRfYXVyb2NfbHMsIC5pZCA9ICJmb2xkIikKbWVhbl92YWxpZF9hdXJvY19kZiA8LSB2YWxpZF9hdXJvY19kZiB8PiAKICBkcGx5cjo6c3VtbWFyaXNlKGRwbHlyOjphY3Jvc3MoLWZvbGQsIH4gbWVhbigueCwgbmEucm0gPSBUUlVFKSkpCgp0aWJibGU6OnRpYmJsZSgKICBgRXN0aW1hdGVkIFRlc3QgQVVST0NgID0gbWVhbl92YWxpZF9hdXJvY19kZiRsb2dpc3RpYywKICBgT2JzZXJ2ZWQgVGVzdCBBVVJPQ2AgPSB0ZXN0X2F1cm9jCikKYGBgCgojIyMgTXVsdGlwbGUgTWV0aG9kcwoKPGRpdiBjbGFzcz0icGFuZWwgcGFuZWwtZGVmYXVsdCBwYWRkZWQtcGFuZWwiPgpXZSBuZXh0IHByb3ZpZGUgYW4gZXhhbXBsZSBvZiBob3cgdG8gcGVyZm9ybSB0aGUgZGF0YSBzcGxpdHRpbmcgc2NoZW1lIHdoZW4gdHJ5aW5nIHRvIGJvdGggdHVuZSBoeXBlcnBhcmFtZXRlcnMgaW4gbW9kZWxzIGFuZCBzZWxlY3RpbmcgdGhlIGJlc3QgbW9kZWwuIAoKKipNb2RlbHMgdW5kZXIgY29uc2lkZXJhdGlvbjoqKgoKLSBMb2dpc3RpYyByZWdyZXNzaW9uCi0gTGFzc28gcmVncmVzc2lvbiAobmVlZHMgdHVuaW5nKQotIFJpZGdlIHJlZ3Jlc3Npb24gKG5lZWRzIHR1bmluZykKLSBSYW5kb20gZm9yZXN0IChubyB0dW5pbmc7IHVzaW5nIGRlZmF1bHQgaHlwZXJwYXJhbWV0ZXJzKQoKTm90ZSB0aGF0IHdpdGhpbiBgZ2xtbmV0Ojpjdi5nbG1uZXRgLCB0aGVyZSBpcyBhbiBpbnRlcmlvciBjcm9zcy12YWxpZGF0aW9uIGxvb3AgdG8gdHVuZSB0aGUgaHlwZXJwYXJhbWV0ZXJzIGZvciBMQVNTTyBhbmQgcmlkZ2UsIGFuZCBieSBleHBsaWNpdGx5IHNldHRpbmcgdGhlIGBmb2xkaWRgLCB3ZSBlbnN1cmUgdGhhdCB0aGUgY3Jvc3MtdmFsaWRhdGlvbiBpcyBkb25lIHVzaW5nIG91ciBjbHVzdGVyZWQgc2FtcGxpbmcgc2NoZW1lIGJ5IGltYWdlIGJsb2NrLiBJZiB0aGUgUiBmdW5jdGlvbiBkb2Vzbid0IGhhdmUgYSBidWlsdC1pbiBDViBvcHRpb24sIHdlIHdvdWxkIG5lZWQgdG8gY29kZSB0aGlzIHVwIG91cnNlbHZlcyAob3IgdXNpbmcgb3RoZXIgZnVuY3Rpb25zIGxpa2UgZnJvbSB0aGUgYGNhcmV0YCBSIHBhY2thZ2UpLgo8L2Rpdj4KCmBgYHtyfQp0cmFpbl9kYXRhIDwtIGNsb3VkX2RhdGEgfD4KICBkcGx5cjo6ZmlsdGVyKCEoaW1hZ2UgJWluJSB0ZXN0X2ltYWdlX2lkeCkpCnRlc3RfZGF0YSA8LSBjbG91ZF9kYXRhIHw+CiAgZHBseXI6OmZpbHRlcihpbWFnZSAlaW4lIHRlc3RfaW1hZ2VfaWR4KQoKIyBldmFsdWF0ZSB2YWxpZGF0aW9uIGVycm9yIGZvciB2YXJpb3VzIG1vZGVscwp2YWxpZF9hdXJvY19scyA8LSBsaXN0KCkKdmFsaWRfcHJlZHNfbHMgPC0gbGlzdCgpCmZvciAoZm9sZCBpbiB1bmlxdWUodHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKSB7CiAgIyBkbyBkYXRhIHNwbGl0CiAgY3ZfdHJhaW5fZGF0YV9hbGwgPC0gdHJhaW5fZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6ZmlsdGVyKGJsb2NrX2lkICE9ICEhZm9sZCkKICBjdl90cmFpbl9kYXRhIDwtIGN2X3RyYWluX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OnNlbGVjdCh0aWR5c2VsZWN0OjphbGxfb2Yoa2VlcF92YXJzKSkKICBjdl92YWxpZF9kYXRhX2FsbCA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpmaWx0ZXIoYmxvY2tfaWQgPT0gISFmb2xkKQogIGN2X3ZhbGlkX2RhdGEgPC0gY3ZfdmFsaWRfZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQogIAogICMgZml0IGxvZ2lzdGljIHJlZ3Jlc3Npb24KICBsb2dfZml0IDwtIGdsbSgKICAgIGJpbmFyeV9sYWJlbCB+IC4sIGRhdGEgPSBjdl90cmFpbl9kYXRhLCBmYW1pbHkgPSAiYmlub21pYWwiCiAgKQogIGxvZ19wcmVkcyA8LSBwcmVkaWN0KGxvZ19maXQsIGN2X3ZhbGlkX2RhdGEsIHR5cGUgPSAicmVzcG9uc2UiKQogIAogICMgZml0IGFuZCBldmFsdWF0ZSBsYXNzbyByZWdyZXNzaW9uCiAgbGFzc29fZml0IDwtIGdsbW5ldDo6Y3YuZ2xtbmV0KAogICAgeCA9IGFzLm1hdHJpeChjdl90cmFpbl9kYXRhIHw+IGRwbHlyOjpzZWxlY3QoLWJpbmFyeV9sYWJlbCkpLAogICAgeSA9IGN2X3RyYWluX2RhdGEkYmluYXJ5X2xhYmVsLAogICAgZmFtaWx5ID0gImJpbm9taWFsIiwKICAgIGFscGhhID0gMSwKICAgIGZvbGRpZCA9IGFzLm51bWVyaWMoYXMuZmFjdG9yKGN2X3RyYWluX2RhdGFfYWxsJGJsb2NrX2lkKSkKICApCiAgbGFzc29fcHJlZHMgPC0gcHJlZGljdCgKICAgIGxhc3NvX2ZpdCwgCiAgICBhcy5tYXRyaXgoY3ZfdmFsaWRfZGF0YSB8PiBkcGx5cjo6c2VsZWN0KC1iaW5hcnlfbGFiZWwpKSwgCiAgICBzID0gImxhbWJkYS5taW4iLCAjIG9yICJsYW1iZGEuMXNlIgogICAgdHlwZSA9ICJyZXNwb25zZSIKICApCiAgCiAgIyBmaXQgYW5kIGV2YWx1YXRlIHJpZGdlIHJlZ3Jlc3Npb24KICByaWRnZV9maXQgPC0gZ2xtbmV0Ojpjdi5nbG1uZXQoCiAgICB4ID0gYXMubWF0cml4KGN2X3RyYWluX2RhdGEgfD4gZHBseXI6OnNlbGVjdCgtYmluYXJ5X2xhYmVsKSksCiAgICB5ID0gY3ZfdHJhaW5fZGF0YSRiaW5hcnlfbGFiZWwsCiAgICBmYW1pbHkgPSAiYmlub21pYWwiLAogICAgYWxwaGEgPSAwLAogICAgZm9sZGlkID0gYXMubnVtZXJpYyhhcy5mYWN0b3IoY3ZfdHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKQogICkKICByaWRnZV9wcmVkcyA8LSBwcmVkaWN0KAogICAgcmlkZ2VfZml0LCAKICAgIGFzLm1hdHJpeChjdl92YWxpZF9kYXRhIHw+IGRwbHlyOjpzZWxlY3QoLWJpbmFyeV9sYWJlbCkpLCAKICAgIHMgPSAibGFtYmRhLm1pbiIsICMgb3IgImxhbWJkYS4xc2UiCiAgICB0eXBlID0gInJlc3BvbnNlIgogICkKICAKICAjIGZpdCByYW5kb20gZm9yZXN0CiAgcmZfZml0IDwtIHJhbmdlcjo6cmFuZ2VyKAogICAgYmluYXJ5X2xhYmVsIH4gLiwgZGF0YSA9IGN2X3RyYWluX2RhdGEsIAogICAgcHJvYmFiaWxpdHkgPSBUUlVFLCB2ZXJib3NlID0gRkFMU0UKICApCiAgcmZfcHJlZHMgPC0gcHJlZGljdChyZl9maXQsIGN2X3ZhbGlkX2RhdGEpJHByZWRpY3Rpb25zWywgMl0KICAKICAjIGV2YWx1YXRlIHByZWRpY3Rpb25zCiAgcHJlZHNfbHMgPC0gbGlzdCgKICAgICJsb2dpc3RpYyIgPSBsb2dfcHJlZHMsCiAgICAibGFzc28iID0gbGFzc29fcHJlZHMsCiAgICAicmlkZ2UiID0gcmlkZ2VfcHJlZHMsCiAgICAicmYiID0gcmZfcHJlZHMKICApCiAgdmFsaWRfYXVyb2NfbHNbW2ZvbGRdXSA8LSBwdXJycjo6bWFwKAogICAgcHJlZHNfbHMsCiAgICB+IHlhcmRzdGljazo6cm9jX2F1Y192ZWMoCiAgICAgIHRydXRoID0gY3ZfdmFsaWRfZGF0YSRiaW5hcnlfbGFiZWwsIAogICAgICBlc3RpbWF0ZSA9IGMoLngpLCAKICAgICAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgogICAgKQogICkgfD4KICAgIGRwbHlyOjpiaW5kX3Jvd3MoLmlkID0gIm1ldGhvZCIpCiAgCiAgIyBzYXZlIGZvbGQgcHJlZGljdGlvbnMgZm9yIGZ1dHVyZSBpbnZlc3RpZ2F0aW9uCiAgdmFsaWRfcHJlZHNfbHNbW2ZvbGRdXSA8LSBjdl92YWxpZF9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpiaW5kX2NvbHMocHJlZHNfbHMpCn0KCiMgZXhhbWluZSB2YWxpZGF0aW9uIGFjY3VyYWN5CnZhbGlkX2F1cm9jX2RmIDwtIGRwbHlyOjpiaW5kX3Jvd3ModmFsaWRfYXVyb2NfbHMsIC5pZCA9ICJmb2xkIikKbWVhbl92YWxpZF9hdXJvY19kZiA8LSB2YWxpZF9hdXJvY19kZiB8PiAKICBkcGx5cjo6c3VtbWFyaXNlKGRwbHlyOjphY3Jvc3MoLWZvbGQsIH4gbWVhbigueCwgbmEucm0gPSBUUlVFKSkpCgojIGV2YWx1YXRlIGJlc3QgbW9kZWwgb24gdGVzdCBzZXQKdHJhaW5fZGF0YSA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQp0ZXN0X2RhdGEgPC0gdGVzdF9kYXRhX2FsbCB8PgogIGRwbHlyOjpzZWxlY3QodGlkeXNlbGVjdDo6YWxsX29mKGtlZXBfdmFycykpCmJlc3RfZml0IDwtIHJhbmdlcjo6cmFuZ2VyKAogIGJpbmFyeV9sYWJlbCB+IC4sIGRhdGEgPSB0cmFpbl9kYXRhLCBwcm9iYWJpbGl0eSA9IFRSVUUsIHZlcmJvc2UgPSBGQUxTRQopCnRlc3RfcHJlZHMgPC0gcHJlZGljdChiZXN0X2ZpdCwgdGVzdF9kYXRhKSRwcmVkaWN0aW9uc1ssIDJdCnRlc3RfYXVyb2MgPC0geWFyZHN0aWNrOjpyb2NfYXVjX3ZlYygKICB0cnV0aCA9IHRlc3RfZGF0YSRiaW5hcnlfbGFiZWwsIAogIGVzdGltYXRlID0gdGVzdF9wcmVkcywgCiAgZXZlbnRfbGV2ZWwgPSAic2Vjb25kIgopCgplcnJfZGYgPC0gbWVhbl92YWxpZF9hdXJvY19kZiB8PgogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIlZhbGlkYXRpb24gJXMgQVVST0MiLCBzdHJpbmdyOjpzdHJfdG9fdGl0bGUoLngpKSkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIGBUZXN0IEFVUk9DYCA9IHRlc3RfYXVyb2MKICApCgp2dGhlbWVzOjpwcmV0dHlfa2FibGUoCiAgdmFsaWRfYXVyb2NfZGYsIGNhcHRpb24gPSAiRm9sZC13aXNlIFZhbGlkYXRpb24gQVVST0MgZm9yIFZhcmlvdXMgTW9kZWxzIgopCnZ0aGVtZXM6OnByZXR0eV9rYWJsZSgKICBlcnJfZGYsIGNhcHRpb24gPSAiT3ZlcmFsbCBQcmVkaWN0aW9uIFBlcmZvcm1hbmNlIgopCmBgYAoKIyMjIEluY2x1ZGluZyBFbmdpbmVlcmVkIEZlYXR1cmVzCgo8ZGl2IGNsYXNzPSJwYW5lbCBwYW5lbC1kZWZhdWx0IHBhZGRlZC1wYW5lbCI+CkVuZ2luZWVyaW5nIGZlYXR1cmVzIGJhc2VkIHVwb24gcHJpb3Igb3IgZG9tYWluIGtub3dsZWRnZSBjYW4gZ3JlYXRseSBpbXByb3ZlIHRoZSBhY2N1cmFjeSBvZiBvdXIgbW9kZWxzLiBJbiBbU2hpIGV0IGFsLiAoMjAwOCldKGh0dHBzOi8vd3d3LmpzdG9yLm9yZy9zdGFibGUvMjc2NDAwODEpLCB0aGUgYXV0aG9ycyBlbmdpbmVlcmVkIHRocmVlIGFkZGl0aW9uYWwgZmVhdHVyZXM6CgotIE5EQUk6IG1lYXN1cmVzIGRpZmZlcmVuY2UgaW4gcmVmbGVjdGFuY2UgdmFsdWVzIGJldHdlZW4gZGlmZmVyZW50IHJhZGlhbmNlIGFuZ2xlcyAob3Igc3BlY3RyYWwgYmFuZHMpCi0gQ09SUjogbWVhc3VyZXMgY29ycmVsYXRpb24gYmV0d2VlbiB0aGUgZGlmZmVyZW50IHJhZGlhbmNlIGFuZ2xlcwotIFNEOiBtZWFzdXJlcyB2YXJpYWJpbGl0eSBpbiByZWZsZWN0YW5jZSB2YWx1ZXMgc3Vycm91bmRpbmcgdGhlIHBpeGVsCjwvZGl2PgoKYGBge3J9CmtlZXBfdmFycyA8LSBjKCJOREFJIiwgIlNEIiwgIkNPUlIiLCBrZWVwX3ZhcnMpCgojIGV2YWx1YXRlIHZhbGlkYXRpb24gZXJyb3IgZm9yIHZhcmlvdXMgbW9kZWxzCnZhbGlkX2F1cm9jX2xzIDwtIGxpc3QoKQp2YWxpZF9wcmVkc19scyA8LSBsaXN0KCkKZm9yIChmb2xkIGluIHVuaXF1ZSh0cmFpbl9kYXRhX2FsbCRibG9ja19pZCkpIHsKICAjIGRvIGRhdGEgc3BsaXQKICBjdl90cmFpbl9kYXRhX2FsbCA8LSB0cmFpbl9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpmaWx0ZXIoYmxvY2tfaWQgIT0gISFmb2xkKQogIGN2X3RyYWluX2RhdGEgPC0gY3ZfdHJhaW5fZGF0YV9hbGwgfD4gCiAgICBkcGx5cjo6c2VsZWN0KHRpZHlzZWxlY3Q6OmFsbF9vZihrZWVwX3ZhcnMpKQogIGN2X3ZhbGlkX2RhdGFfYWxsIDwtIHRyYWluX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OmZpbHRlcihibG9ja19pZCA9PSAhIWZvbGQpCiAgY3ZfdmFsaWRfZGF0YSA8LSBjdl92YWxpZF9kYXRhX2FsbCB8PiAKICAgIGRwbHlyOjpzZWxlY3QodGlkeXNlbGVjdDo6YWxsX29mKGtlZXBfdmFycykpCiAgCiAgIyBmaXQgbG9naXN0aWMgcmVncmVzc2lvbgogIGxvZ19maXQgPC0gZ2xtKAogICAgYmluYXJ5X2xhYmVsIH4gLiwgZGF0YSA9IGN2X3RyYWluX2RhdGEsIGZhbWlseSA9ICJiaW5vbWlhbCIKICApCiAgbG9nX3ByZWRzIDwtIHByZWRpY3QobG9nX2ZpdCwgY3ZfdmFsaWRfZGF0YSwgdHlwZSA9ICJyZXNwb25zZSIpCiAgCiAgIyBmaXQgYW5kIGV2YWx1YXRlIGxhc3NvIHJlZ3Jlc3Npb24KICBsYXNzb19maXQgPC0gZ2xtbmV0Ojpjdi5nbG1uZXQoCiAgICB4ID0gYXMubWF0cml4KGN2X3RyYWluX2RhdGEgfD4gZHBseXI6OnNlbGVjdCgtYmluYXJ5X2xhYmVsKSksCiAgICB5ID0gY3ZfdHJhaW5fZGF0YSRiaW5hcnlfbGFiZWwsCiAgICBmYW1pbHkgPSAiYmlub21pYWwiLAogICAgYWxwaGEgPSAxLAogICAgZm9sZGlkID0gYXMubnVtZXJpYyhhcy5mYWN0b3IoY3ZfdHJhaW5fZGF0YV9hbGwkYmxvY2tfaWQpKQogICkKICBsYXNzb19wcmVkcyA8LSBwcmVkaWN0KAogICAgbGFzc29fZml0LCAKICAgIGFzLm1hdHJpeChjdl92YWxpZF9kYXRhIHw+IGRwbHlyOjpzZWxlY3QoLWJpbmFyeV9sYWJlbCkpLCAKICAgIHMgPSAibGFtYmRhLm1pbiIsICMgb3IgImxhbWJkYS4xc2UiCiAgICB0eXBlID0gInJlc3BvbnNlIgogICkKICAKICAjIGZpdCBhbmQgZXZhbHVhdGUgcmlkZ2UgcmVncmVzc2lvbgogIHJpZGdlX2ZpdCA8LSBnbG1uZXQ6OmN2LmdsbW5ldCgKICAgIHggPSBhcy5tYXRyaXgoY3ZfdHJhaW5fZGF0YSB8PiBkcGx5cjo6c2VsZWN0KC1iaW5hcnlfbGFiZWwpKSwKICAgIHkgPSBjdl90cmFpbl9kYXRhJGJpbmFyeV9sYWJlbCwKICAgIGZhbWlseSA9ICJiaW5vbWlhbCIsCiAgICBhbHBoYSA9IDAsCiAgICBmb2xkaWQgPSBhcy5udW1lcmljKGFzLmZhY3Rvcihjdl90cmFpbl9kYXRhX2FsbCRibG9ja19pZCkpCiAgKQogIHJpZGdlX3ByZWRzIDwtIHByZWRpY3QoCiAgICByaWRnZV9maXQsIAogICAgYXMubWF0cml4KGN2X3ZhbGlkX2RhdGEgfD4gZHBseXI6OnNlbGVjdCgtYmluYXJ5X2xhYmVsKSksIAogICAgcyA9ICJsYW1iZGEubWluIiwgIyBvciAibGFtYmRhLjFzZSIKICAgIHR5cGUgPSAicmVzcG9uc2UiCiAgKQogIAogICMgZml0IHJhbmRvbSBmb3Jlc3QKICByZl9maXQgPC0gcmFuZ2VyOjpyYW5nZXIoCiAgICBiaW5hcnlfbGFiZWwgfiAuLCBkYXRhID0gY3ZfdHJhaW5fZGF0YSwgCiAgICBwcm9iYWJpbGl0eSA9IFRSVUUsIHZlcmJvc2UgPSBGQUxTRQogICkKICByZl9wcmVkcyA8LSBwcmVkaWN0KHJmX2ZpdCwgY3ZfdmFsaWRfZGF0YSkkcHJlZGljdGlvbnNbLCAyXQogIAogICMgZXZhbHVhdGUgcHJlZGljdGlvbnMKICBwcmVkc19scyA8LSBsaXN0KAogICAgImxvZ2lzdGljIiA9IGxvZ19wcmVkcywKICAgICJsYXNzbyIgPSBsYXNzb19wcmVkcywKICAgICJyaWRnZSIgPSByaWRnZV9wcmVkcywKICAgICJyZiIgPSByZl9wcmVkcwogICkKICB2YWxpZF9hdXJvY19sc1tbZm9sZF1dIDwtIHB1cnJyOjptYXAoCiAgICBwcmVkc19scywKICAgIH4geWFyZHN0aWNrOjpyb2NfYXVjX3ZlYygKICAgICAgdHJ1dGggPSBjdl92YWxpZF9kYXRhJGJpbmFyeV9sYWJlbCwgCiAgICAgIGVzdGltYXRlID0gYygueCksIAogICAgICBldmVudF9sZXZlbCA9ICJzZWNvbmQiCiAgICApCiAgKSB8PgogICAgZHBseXI6OmJpbmRfcm93cyguaWQgPSAibWV0aG9kIikKICAKICAjIHNhdmUgZm9sZCBwcmVkaWN0aW9ucyBmb3IgZnV0dXJlIGludmVzdGlnYXRpb24KICB2YWxpZF9wcmVkc19sc1tbZm9sZF1dIDwtIGN2X3ZhbGlkX2RhdGFfYWxsIHw+IAogICAgZHBseXI6OmJpbmRfY29scyhwcmVkc19scykKfQoKIyBleGFtaW5lIHZhbGlkYXRpb24gYWNjdXJhY3kKdmFsaWRfYXVyb2NfZGYgPC0gZHBseXI6OmJpbmRfcm93cyh2YWxpZF9hdXJvY19scywgLmlkID0gImZvbGQiKQptZWFuX3ZhbGlkX2F1cm9jX2RmIDwtIHZhbGlkX2F1cm9jX2RmIHw+IAogIGRwbHlyOjpzdW1tYXJpc2UoZHBseXI6OmFjcm9zcygtZm9sZCwgfiBtZWFuKC54LCBuYS5ybSA9IFRSVUUpKSkKCiMgZXZhbHVhdGUgYmVzdCBtb2RlbCBvbiB0ZXN0IHNldAp0cmFpbl9kYXRhIDwtIHRyYWluX2RhdGFfYWxsIHw+IAogIGRwbHlyOjpzZWxlY3QodGlkeXNlbGVjdDo6YWxsX29mKGtlZXBfdmFycykpCnRlc3RfZGF0YSA8LSB0ZXN0X2RhdGFfYWxsIHw+CiAgZHBseXI6OnNlbGVjdCh0aWR5c2VsZWN0OjphbGxfb2Yoa2VlcF92YXJzKSkKYmVzdF9maXQgPC0gcmFuZ2VyOjpyYW5nZXIoCiAgYmluYXJ5X2xhYmVsIH4gLiwgZGF0YSA9IHRyYWluX2RhdGEsIHByb2JhYmlsaXR5ID0gVFJVRSwgdmVyYm9zZSA9IEZBTFNFCikKdGVzdF9wcmVkcyA8LSBwcmVkaWN0KGJlc3RfZml0LCB0ZXN0X2RhdGEpJHByZWRpY3Rpb25zWywgMl0KdGVzdF9hdXJvYyA8LSB5YXJkc3RpY2s6OnJvY19hdWNfdmVjKAogIHRydXRoID0gdGVzdF9kYXRhJGJpbmFyeV9sYWJlbCwgCiAgZXN0aW1hdGUgPSB0ZXN0X3ByZWRzLCAKICBldmVudF9sZXZlbCA9ICJzZWNvbmQiCikKCm5ld19lcnJfZGYgPC0gbWVhbl92YWxpZF9hdXJvY19kZiB8PgogIGRwbHlyOjpyZW5hbWVfd2l0aCh+IHNwcmludGYoIlZhbGlkYXRpb24gJXMgQVVST0MiLCBzdHJpbmdyOjpzdHJfdG9fdGl0bGUoLngpKSkgfD4gCiAgZHBseXI6Om11dGF0ZSgKICAgIGBUZXN0IEFVUk9DYCA9IHRlc3RfYXVyb2MKICApCm5ld19lcnJfZGYgPC0gbGlzdCgKICAiUmF3IEZlYXR1cmVzIE9ubHkiID0gZXJyX2RmLAogICJJbmNsdWRpbmcgRW5naW5lZXJlZCBGZWF0dXJlcyIgPSBuZXdfZXJyX2RmCikgfD4gCiAgZHBseXI6OmJpbmRfcm93cyguaWQgPSAiRmVhdHVyZSBTZXQiKQoKdnRoZW1lczo6cHJldHR5X2thYmxlKAogIHZhbGlkX2F1cm9jX2RmLCBjYXB0aW9uID0gIkZvbGQtd2lzZSBWYWxpZGF0aW9uIEFVUk9DIGZvciBWYXJpb3VzIE1vZGVscyIKKQp2dGhlbWVzOjpwcmV0dHlfa2FibGUobmV3X2Vycl9kZikKYGBgCgojIyMjIFBvc3QtaG9jIGludmVzdGlnYXRpb25zCgpMZXQncyBsb29rIGF0IHRoZSBoZWxkLW91dCBmb2xkcyB3aGVyZSB0aGUgbWV0aG9kcyBkaWRuJ3QgcGVyZm9ybSBzbyB3ZWxsIGFuZCBjb21wYXJlIHRoZSBwcmVkaWN0aW9ucyBhY3Jvc3MgbWV0aG9kcy4KCmBgYHtyfQpmb2xkIDwtICIxMCIKcHJlZHNfZGYgPC0gdmFsaWRfcHJlZHNfbHNbW2ZvbGRdXQpwbG90X3ZhcnMgPC0gYygKICAiYmluYXJ5X2xhYmVsIiwgImxvZ2lzdGljIiwgImxhc3NvIiwgInJpZGdlIiwgInJmIiwKICAiREYiLCAiQ0YiLCAiQkYiLCAiQUYiLCAiQU4iLCAiTkRBSSIsICJTRCIsICJDT1JSIgopCnBsdF9scyA8LSBsaXN0KCkKZm9yICh2YXIgaW4gcGxvdF92YXJzKSB7CiAgcGx0X2xzW1t2YXJdXSA8LSBwbG90X2Nsb3VkX2RhdGEocHJlZHNfZGYsIHZhcikKfQpwbHQgPC0gcGF0Y2h3b3JrOjp3cmFwX3Bsb3RzKHBsdF9scywgbmNvbCA9IDUpCnBsdAoKIyBwbHQgPC0gcGxvdF9wYWlycygKIyAgIHByZWRzX2RmLCBjb2x1bW5zID0gYygibG9naXN0aWMiLCAibGFzc28iLCAicmlkZ2UiLCAicmYiKSwgCiMgICBwb2ludF9zaXplID0gMC4xLCBjb2xvciA9IHByZWRzX2RmJGJpbmFyeV9sYWJlbAojICkgKwojICAgZ2dwbG90Mjo6c2NhbGVfY29sb3JfbWFudWFsKHZhbHVlcyA9IGNsb3VkX2NvbG9ycykgKwojICAgZ2dwbG90Mjo6c2NhbGVfZmlsbF9tYW51YWwodmFsdWVzID0gY2xvdWRfY29sb3JzCiMgKQojIHBsdApgYGAKCgojIEludGVycHJldGF0aW9ucwoKVG8gYmUgY29udGludWVkLi4uCg==